# -*- coding: utf-8 -*-
import scrapy
import re
import json


class YahooSpiderSpider(scrapy.Spider):
    name = 'yahoo_spider'
    allowed_domains = ['yahoo.com']
    search_msg = 'BABA'
    base_url = "https://finance.yahoo.com/quote/%s?p=%s"
    # 入口地址收集ids
    ids_url = "https://video-api.yql.yahoo.com/v1/video/alias/channels/finance-ticker/ids?lang=en-US&region=US&site=finance&count=10&dev_type=desktop&expn=qsp&tickers=%s&corsDomain=finance.yahoo.com"
    # 详情地址
    content_url = '''
    https://finance.yahoo.com/_finance_doubledown/api/resource/content;fetchNewAttribution=true;getDetailView=true;getFullLcp=false;imageResizer=null;relatedContent={"enabled":true};site=finance;useSlingstoneLcp=true;uuids=[%s]?bkt=["findmcontrol1","fmw-display-ads-on"]&device=smartphone&feature=canvassOffnet,enableChartiqFeedback,enableCrypto,newContentAttribution,relatedVideoFeature,videoNativePlaylist,enableESG,livecoverage,enableSingleRail,enableSKTVLrec,enableTradeItPromo&intl=us&lang=en-US&partner=none&prid=568f1vdd87umk&region=US&site=finance&tz=America/Los_Angeles&ver=0.102.1112&returnMeta=true
    '''
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"}

    def start_requests(self):
        url = self.ids_url % self.search_msg
        # url = "http://www.baidu.com"
        yield scrapy.Request(url=url, callback=self.parse, headers=self.headers, dont_filter=True)

    def parse(self, response):
        html = response.body
        with open("./inter_game_spider/test/yahoo.html", 'wb') as fs:
            fs.write(html)
        msg_dict = json.loads(html)
        mediaItems = msg_dict["channel"]["result"][0]["mediaItems"]
        ids = []
        pubtimes = []
        for item in mediaItems:
            id = item["id"]
            ids.append(id)
            publish_time = item["publish_time"]
            pubtimes.append(pubtimes)
        pass

    def get_msg_by_pat(self, data, pat, mode=re.S):
        result = []
        result = re.compile(pat, mode).findall(data)
        if len(result) < 1:
            return None
        return result
'''
https://finance.yahoo.com/_finance_doubledown/api/resource/content;
fetchNewAttribution=true;getDetailView=true;getFullLcp=false;imageResizer=null;
relatedContent=%7B%22enabled%22%3Atrue%7D;site=finance;useSlingstoneLcp=true;
uuids=%5B%22FINR-77b102aa-1fcf-3309-83f9-a2a320828a82%22%2C%22FINR-1dcbd061-7835-3f63-9bec-92a9dc5f4232%22%2C%22FIN-cd2aa721-1ca5-3f35-8f48-c582112a6369%22%2C%22FINR-b65a8d6c-b6a4-3476-b6ca-b24e77e9e020%22%2C%22FIN-73f62d40-8801-35e1-8873-e73e16289da9%22%2C%22FINR-6f60805d-0e96-3659-a606-fb9c6c0bc15e%22%5D?bkt=%5B%22fin-strm-vespa1%22%2C%22fn-sa-4%22%2C%22fdw-port-ldrb%22%5D
&device=desktop
&feature=canvassOffnet%2CenableChartiqFeedback%2CenableCrypto%2CnewContentAttribution%2CrelatedVideoFeature%2CvideoNativePlaylist%2CenableESG%2Clivecoverage%2CenableSingleRail%2CenableSKTVLrec%2ChostedVespa%2CretirementCalc
&intl=us&lang=en-US&partner=none&prid=6ulot9td87cba&region=US&site=finance&tz=Asia%2FShanghai&ver=0.102.1112&returnMeta=true
'''

'''
https://finance.yahoo.com/_finance_doubledown/api/resource/content;
fetchNewAttribution=true;getDetailView=true;getFullLcp=false;imageResizer=null;
relatedContent=%7B%22enabled%22%3Atrue%7D;site=finance;useSlingstoneLcp=true;
uuids=%5B%22FINR-d5d68d73-2425-3059-8ba6-55f46f2d55c1%22%2C%22FINR-25e66932-e3f7-3488-9cf6-e62a9288cbee%22%2C%22FIN-aa089b15-8a2b-3c50-97fb-817ab724aabb%22%2C%22FINR-ecc188ce-7188-3f64-874d-9ec838642f03%22%2C%22FINR-f6b22884-8772-3f34-8856-c5649e89bd7c%22%2C%22FINR-ec09f9b4-8569-353e-b59f-45d59d615576%22%2C%22FINR-9b3f1eb8-9ec9-30db-8a1d-c8cf9aad4e11%22%2C%22FINR-046b432e-d8f7-3102-84d4-5ffbd47bbe36%22%2C%22FINR-49faad0f-430b-386c-b2b1-e21ae5142306%22%5D
?bkt=%5B%22fin-strm-vespa1%22%2C%22fn-sa-4%22%2C%22fdw-port-ldrb%22%5D
&device=desktop
&feature=canvassOffnet%2CenableChartiqFeedback%2CenableCrypto%2CnewContentAttribution%2CrelatedVideoFeature%2CvideoNativePlaylist%2CenableESG%2Clivecoverage%2CenableSingleRail%2CenableSKTVLrec%2ChostedVespa%2CretirementCalc
&intl=us&lang=en-US
&partner=none
&prid=dafq995d87def
&region=US
&site=finance
&tz=Asia%2FShanghai
&ver=0.102.1112
&returnMeta=true
'''